package com.shujia.spark.core

import org.apache.spark.{SparkConf, SparkContext}

object Demo22Args {
  def main(args: Array[String]): Unit = {

    /**
      * spark运行参数设置
      * 1、在代码中设置   优先级最高
      * 2、在提交任务的时候设置
      */

    val conf = new SparkConf()

    //第一种方式，在代码中设置
    conf.setAppName("app")
    conf.setMaster("local")
    conf.set("spark.shuffle.file.buffer", "32k")
    conf.set("spark.reducer.maxSizeInFlight", "96m")
    conf.set("spark.shuffle.io.maxRetries", "10")
    conf.set("spark.shuffle.io.retryWait", "60s")
    conf.set("spark.shuffle.memoryFraction", "0.3")
    conf.set("spark.shuffle.sort.bypassMergeThreshold", "1000")

    val sc = new SparkContext(conf)

    //第二中方式
    /**
      * spark-submit --master yarn-client --conf  spark.shuffle.file.buffer=32k  --conf spark.reducer.maxSizeInFlight=96m
      *
      */

  }

}
